from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#需要自己先手动注册一个账号
my_username = '17788559601'
my_password = 'Roottoor@2'

# 1. 打开网站
driver = webdriver.Chrome()
driver.get("https://www.epubit.com/")

# 2. 点击登录

# 等待按钮可以被点击再继续执行，最多等待60秒
wait = WebDriverWait(driver, 60)
element = wait.until(EC.presence_of_element_located(
    (By.XPATH, '//i[text()="登录"]')))

#仍然需要等待2秒，让相关的JavaScript完成渲染
time.sleep(2)

login_btn = driver.find_element(By.XPATH, '//i[text()="登录"]')
login_btn.click()

# 3. 输入账号密码
# 等待输入框出现
wait = WebDriverWait(driver, 60)
element = wait.until(EC.presence_of_element_located(
    (By.XPATH, '//input[@id="username"]')))

input_username = driver.find_element(By.XPATH, '//input[@id="username"]')
input_username.send_keys(my_username)

input_password = driver.find_element(By.XPATH, '//input[@id="password"]')
input_password.send_keys(my_password)

# 4. 提交登录
submit_btn = driver.find_element(By.XPATH, '//input[@id="passwordLoginBtn"]')
submit_btn.click()

# 5. 循环抓取
course_data_list = [] #用一个二维数组存放课程信息
driver.get('https://www.epubit.com/course')

# 等到我们要的数据（用CSS选择器确定）出现再继续执行，但最多等待60秒
wait = WebDriverWait(driver, 60)
element = wait.until(EC.presence_of_element_located(
    (By.CSS_SELECTOR, ".course-list a")))

#先获取第一页的数据
item_list = driver.find_elements_by_css_selector('.course-list a')
while True:
    for item in item_list:
        link = item.get_attribute('href')
        img = item.find_element_by_tag_name('img').get_attribute('src')
        title = item.find_element_by_class_name('list-title').text
        price = item.find_element_by_class_name('price').text
        info = item.find_element_by_class_name('info').text
        course_data_list.append([link, img, title, price, info])
        print('==> {}'.format(title))

    next_page_btn = driver.find_element_by_css_selector('button.btn-next')
    #如果没有下一页了，跳出循环
    if 'disabled' in next_page_btn.get_attribute('class'):
        print('last page')
        break
    else:
        next_page_btn.click()
        print('waiting...')
        # 这里用presence_of_element_located不再有用，因为在元素在上一页已经存在了，所以这里简单粗暴的等待3秒
        time.sleep(3)
        item_list = driver.find_elements_by_css_selector('.course-list a')

# 打印共抓取的课程数量
print('total {} items'.format(len(course_data_list)))
driver.close()
